<!DOCTYPE html>



  


<html class="theme-next muse use-motion" lang="zh-Hans">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">









<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />
















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />







<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.4" rel="stylesheet" type="text/css" />


  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=5.1.4">


  <link rel="mask-icon" href="/images/logo.svg?v=5.1.4" color="#222">





  <meta name="keywords" content="Hexo, NexT" />










<meta name="description" content="笔者比较“悲催”，临近年末由笔者维护的生产MQ集群中的一台物理机内存故障导致操作系统异常重启，持续10分钟中出现众多的应用发送客户端出现发送消息超时，导致事故并定性为S1，笔者的“年终奖”。。。">
<meta property="og:type" content="article">
<meta property="og:title" content="从年末生产故障解锁RocketMQ集群部署最佳实践">
<meta property="og:url" content="https://www.codingw.net/posts/8956924f.html">
<meta property="og:site_name" content="中间件兴趣圈">
<meta property="og:description" content="笔者比较“悲催”，临近年末由笔者维护的生产MQ集群中的一台物理机内存故障导致操作系统异常重启，持续10分钟中出现众多的应用发送客户端出现发送消息超时，导致事故并定性为S1，笔者的“年终奖”。。。">
<meta property="og:locale">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20201227093130257.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20201227093154348.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/2020122709322326.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20201227093237477.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20201227093255965.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20201227093317856.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20201227093337453.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20201227093642967.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">
<meta property="article:published_time" content="2020-12-27T02:06:35.000Z">
<meta property="article:modified_time" content="2021-04-26T13:21:17.423Z">
<meta property="article:author" content="中间件兴趣圈">
<meta property="article:tag" content="中间件">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://img-blog.csdnimg.cn/20201227093130257.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '',
    scheme: 'Muse',
    version: '5.1.4',
    sidebar: {"position":"left","display":"post","offset":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="https://www.codingw.net/posts/8956924f.html"/>





  <title>从年末生产故障解锁RocketMQ集群部署最佳实践 | 中间件兴趣圈</title>
  








<meta name="generator" content="Hexo 5.4.0"></head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">中间件兴趣圈</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle">微信搜『中间件兴趣圈』，回复『Java』获取200本优质电子书</p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-question-circle"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-question-circle"></i> <br />
            
            归档
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="https://www.codingw.net/posts/8956924f.html">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="中间件兴趣圈">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">从年末生产故障解锁RocketMQ集群部署最佳实践</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2020-12-27T10:06:35+08:00">
                2020-12-27
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/rocketmq/" itemprop="url" rel="index">
                    <span itemprop="name">rocketmq</span>
                  </a>
                </span>

                
                
                  ， 
                
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/rocketmq/%E6%95%85%E9%9A%9C%E5%88%86%E6%9E%90/" itemprop="url" rel="index">
                    <span itemprop="name">故障分析</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          
          
             <span id="/posts/8956924f.html" class="leancloud_visitors" data-flag-title="从年末生产故障解锁RocketMQ集群部署最佳实践">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数&#58;</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

          
            <span class="post-meta-divider">|</span>
            <span class="page-pv"><i class="fa fa-file-o"></i>
            <span class="busuanzi-value" id="busuanzi_value_page_pv" ></span>次
            </span>
          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <div id="vip-container"><p>笔者比较“悲催”，临近年末由笔者维护的生产MQ集群中的一台物理机内存故障导致操作系统异常重启，持续10分钟中出现众多的应用发送客户端出现发送消息超时，导致事故并定性为S1，笔者的“年终奖”。。。</p>
<span id="more"></span>

<h2 id="1、故障描述"><a href="#1、故障描述" class="headerlink" title="1、故障描述"></a>1、故障描述</h2><p>RocketMQ 集群采取的部署架构为2主2从，其部署架构如下图所示：<br>[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传<br><img src="https://img-blog.csdnimg.cn/20201227093130257.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>其部署架构中一个非常明显的特点是一台物理机上分别部署了 nameserver，broker 两个进程。</p>
<p>其中一台机器(192.168.3.100)的内存出现故障，导致机器重启，但Linux操作系统由于重启需要自检等因素，整个重启过程竟然持续了将近10分钟，客户端的发送超时持续10分钟，这显然是不能接受的！！！</p>
<p>RocketMQ的高可用设计何在？接下来我们将详细介绍其分析过程。</p>
<h2 id="2、故障分析"><a href="#2、故障分析" class="headerlink" title="2、故障分析"></a>2、故障分析</h2><p>当得知一台机器故障导致故障持续10分钟，我的第一反应是不应该呀，因为 RocketMQ 集群是分布式部署架构，天然支持故障发现与故障恢复，消息发送客户端能自动感知 Broker 异常的的时间绝对不会超过10分钟，那故障又是怎么发生的呢？</p>
<p>首先我们先来回顾一下RocketMQ的路由注册与发现机制。</p>
<h4 id="2-1-RocketMQ路由注册与剔除机制"><a href="#2-1-RocketMQ路由注册与剔除机制" class="headerlink" title="2.1 RocketMQ路由注册与剔除机制"></a>2.1 RocketMQ路由注册与剔除机制</h4><p><img src="https://img-blog.csdnimg.cn/20201227093154348.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>其路由注册、剔除机制说明如下：</p>
<ul>
<li>集群中所有Broker每隔30s向集群中所有的NameServer发送心跳包，注册Topic路由信息。</li>
<li>NameServer在收到Broker端的心跳包时首先会更新路由表，并记录收到心跳包的时间。</li>
<li>NameServer会启动一个定时任务每10s会扫描Broker，如果Nameserver连续120s未收到Broker的心跳包，会判定Broker已下线，将从路由表中将该Broker移除。</li>
<li>如果Nameserver与Broker端的长连接断开，NameServer会立即感知Broker下线并从路由表中将该Broker移除。</li>
<li>消息客户端(消息发送者、消息消费者)在任意时刻只会和其中一台NameServer建立连接，并每隔30s向NameServer查询路由信息，如果查询到结果会更新发送者的本地路由信息。</li>
</ul>
<p>从上述的路由注册、剔除机制来看，当一台Broker服务器宕机，消息发送者感知路由信息发生变化需要的时间是多长呢？</p>
<p>分如下两种情况分别讨论：</p>
<ul>
<li>NameServer与Broker服务器TCP连接断开，此时NameServer能立即感知路由信息变化，将其从路由表中移除，从而消息发送端应该在30s左右就能感知路由发送变化，在此30s内在发送端会出现消息发送失败,但结合发送规避机制，并不会对发送方带来重大故障，可接受。</li>
<li>如果NameServer与Broker服务器的TCP连接未断开，但Broker已无法提供服务(例如假死)，此时NameServer需要120s才能感知Broker宕机，此时消息发送端最多需要150s才能感知其路由信息的变化。</li>
</ul>
<p><strong>但问题来了，为什么在生产实际过程中一台Broker由于内存故障重启,10分钟后重启成功后业务才恢复，即业务才真正感知Broker宕机呢？</strong></p>
<p>既然出现了，我们就需要对其进行分析，给出解决方案，避免不会在生产环境出现同类型的错误。</p>
<h4 id="2-2-故障排查经过"><a href="#2-2-故障排查经过" class="headerlink" title="2.2 故障排查经过"></a>2.2 故障排查经过</h4><p>先查询客户端的日志(/home/{user}/logs/rocketmqlogs/rocketmq_client.log)，从中可以看到从客户端第一次报消息发送超时的时间是14:44，其日志输出如下：<br><img src="https://img-blog.csdnimg.cn/2020122709322326.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>由于192.168.3.100机器内存故障，故首先去查看该集群中其他nameserver中的日志，看正常机器中的NameServer感知broker-a故障的时长，其日志如下所示：<br><img src="https://img-blog.csdnimg.cn/20201227093237477.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>从中可以看出192.138.3.101的nameserver基本在2分钟左右才感知其宕机，即虽然机器在重启，但可能由于操作系统要做硬件自检等其他原因，TCP连接并未断开，故nameserver在120s后才感知其宕机，从路由信息表中将该broker移除，那按照路由剔除机制，客户端应该在150秒的时间内感知其变化，那为什么没感知呢？</p>
<p>继续查看客户端路由信息，查看客户端感知路由信息发生变化的时间点，如下图所示：<br><img src="https://img-blog.csdnimg.cn/20201227093255965.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>从客户端日志来看，客户端在14:53:46才感知其变化，这又是为什么呢？</p>
<p><strong>原来客户端在更新路由信息时报超时异常，其截图如下所示：</strong>：<br><img src="https://img-blog.csdnimg.cn/20201227093317856.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>从发生故障到故障恢复期间，客户端一直尝试从已发生故障的NameServer去更新路由信息，但一直返回超时，这样就导致了客户端一直无法获取最新的路由信息，故一直无法感知已宕机的Broker。</p>
<p><strong>从日志分析来看，到目前来说就比较明朗了，客户端之所有没有在120s之内感知其路由信息的变化，是因为客户端一直尝试从已宕机的nameserver去更新路由信息，但由于一直无法请求成功，故客户端的缓存路由信息一直无法得到更新，造成了上面的现象</strong></p>
<p>那问题来了，按照我们对RocketMQ的认识，NameServer宕机，客户端会自动去从nameserver列表中选择下一个nameserver，<strong>那为什么这里并没有发生nameserver切换</strong>，而是等到14:53才切换呢？</p>
<p>接下来我们将目光投向NameServer的切换代码，其代码片段如下图所示：<br><img src="https://img-blog.csdnimg.cn/20201227093337453.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>上图中的几个关键分析如下：</p>
<ul>
<li>客户端能通过缓存中的连接发送RPC请求的前提条件是channel的isActive方法返回true，即底层TCP连接处于激活状态。</li>
<li>在客户端向服务端发起RPC请求时，如果出现非超时类异常，会执行closeChannel方法，该方法会关闭连接并从连接缓存表中移除，这个非常关键，因为在切换NameServer时如果缓存中存在连接并连接处于激活状态，就不会切换nameserver。</li>
<li>如果发送RPC超时，rocketmq会根据clientCloseSocketIfTimeout参数来决定是否关闭连接，但遗憾的是该参数默认为false，并且并未提供修改的入口。</li>
</ul>
<p>那问题分析到这里，已经非常明了，由于机器内存故障触发重启并且重启前需要自检等因素，造成nameserver,broker无法再处理请求但底层TCP连接并未断开，导致发生超时错误，但客户端并不会关闭与故障机器nameserver的TCP连接，导致无法切换，等到机器重新启动后，TCP连接断开，故障机器重启完成后感知路由信息变化，故障恢复。</p>
<p><strong>经过上面的问题分析，其故障原因如下</strong>：<br>192.168.3.100机器在内存故障后重启，整个重启耗时10分钟，并且在重启过程中TCP连接未断开，192.168.3.101 nameserver在故障发送时2分钟左右才感知路由变化，但部分客户端时连接192.168.3.100的nameserver，客户端尝试从该nameserver查询路由信息，但一直返回超时，<strong>由于没有关闭连接，导致客户端并不会切换到</strong>3.101的nameserver，直到客户端与nameserver的TCP连接断开后，切换到另外一个3.101的nameserver，故障在指定时间内得以恢复。</p>
<p><strong>根本原因：其实是nameserver的假死导致路由信息无法更新。</strong></p>
<h2 id="3、最佳实践"><a href="#3、最佳实践" class="headerlink" title="3、最佳实践"></a>3、最佳实践</h2><p>经过上面的故障，个人觉得nameserver不应该与broker部署在一起，如果nameserver与broker并不部署在一起，上面的问题能得到有效避免，其部署架构如下图所示：<br><img src="https://img-blog.csdnimg.cn/20201227093642967.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3ByZXN0aWdlZGluZw==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br><strong>这样的部署架构如果面对上面的故障，Broker假死的情况，能有效避免吗？答案是可以的。</strong></p>
<p>如果192.168.3.100的broker假死，那么3.110,3.111的nameserver都能在2分钟内感知broker-a宕机，然后客户端能成功从nameserver处获得最新的路由信息，如果nameserver假死，出现超时错误，只要broker不宕机，则通过缓存，还是能正常工作的，<strong>但如果nanmeserver,broker一起假死，则上述架构还是无法规避上面的问题</strong>。</p>
<p><strong>故本次的最佳实践主要包含如下两条</strong>：<br>1、nameserver与broker一定要分开部署，进行隔离。<br>2、nameserver与客户端的连接，应该在超时后，关闭连接，触发nameserver漂移，需要修改源码。</p>
</div>

			<script src="https://my.openwrite.cn/js/readmore.js" type="text/javascript"></script>
			<script>
			var isMobile = navigator.userAgent.match(/(phone|pad|pod|iPhone|iPod|ios|iPad|Android|Mobile|BlackBerry|IEMobile|MQQBrowser|JUC|Fennec|wOSBrowser|BrowserNG|WebOS|Symbian|Windows Phone)/i);
			if (!isMobile) {
			    var btw = new BTWPlugin();
			    btw.init({
			        "id": "vip-container",
			        "blogId": "18019-1573088808868-542",
			        "name": "中间件兴趣圈",
			        "qrcode": "https://img-blog.csdnimg.cn/20190314214003962.jpg",
			        "keyword": "more"
			    });
			}
			</script>
		
      
    </div>
    
    
    

    

    

    

    <footer class="post-footer">
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/posts/5cfc78e.html" rel="next" title="源码阅读方法、误区与三种境界">
                <i class="fa fa-chevron-left"></i> 源码阅读方法、误区与三种境界
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/posts/cd43c662.html" rel="prev" title="图文并茂剖析Netty编解码以及背后的设计理念">
                图文并茂剖析Netty编解码以及背后的设计理念 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          

  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <p class="site-author-name" itemprop="name"></p>
              <p class="site-description motion-element" itemprop="description"></p>
          </div>

          <nav class="site-state motion-element">

            
              <div class="site-state-item site-state-posts">
              
                <a href="/archives/">
              
                  <span class="site-state-item-count">139</span>
                  <span class="site-state-item-name">日志</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-categories">
                <a href="/categories/index.html">
                  <span class="site-state-item-count">18</span>
                  <span class="site-state-item-name">分类</span>
                </a>
              </div>
            

            

          </nav>

          

          

          
          

          
          

          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#1%E3%80%81%E6%95%85%E9%9A%9C%E6%8F%8F%E8%BF%B0"><span class="nav-number">1.</span> <span class="nav-text">1、故障描述</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#2%E3%80%81%E6%95%85%E9%9A%9C%E5%88%86%E6%9E%90"><span class="nav-number">2.</span> <span class="nav-text">2、故障分析</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#2-1-RocketMQ%E8%B7%AF%E7%94%B1%E6%B3%A8%E5%86%8C%E4%B8%8E%E5%89%94%E9%99%A4%E6%9C%BA%E5%88%B6"><span class="nav-number">2.0.1.</span> <span class="nav-text">2.1 RocketMQ路由注册与剔除机制</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#2-2-%E6%95%85%E9%9A%9C%E6%8E%92%E6%9F%A5%E7%BB%8F%E8%BF%87"><span class="nav-number">2.0.2.</span> <span class="nav-text">2.2 故障排查经过</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#3%E3%80%81%E6%9C%80%E4%BD%B3%E5%AE%9E%E8%B7%B5"><span class="nav-number">3.</span> <span class="nav-text">3、最佳实践</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">中间件兴趣圈</span>

  
</div>


  <div class="powered-by">由 <a class="theme-link" target="_blank" href="https://hexo.io">Hexo</a> 强力驱动</div>



  <span class="post-meta-divider">|</span>



  <div class="theme-info">主题 &mdash; <a class="theme-link" target="_blank" href="https://github.com/iissnan/hexo-theme-next">NexT.Muse</a> v5.1.4</div>




        
<div class="busuanzi-count">
  <script async src="https://dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js"></script>

  
    <span class="site-uv">
      <i class="fa fa-user"></i>
      <span class="busuanzi-value" id="busuanzi_value_site_uv"></span>
      
    </span>
  

  
    <span class="site-pv">
      <i class="fa fa-eye"></i>
      <span class="busuanzi-value" id="busuanzi_value_site_pv"></span>
      
    </span>
  
</div>








        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  
    <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>
  

  
  
    <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>
  

  
  
    <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>
  


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.4"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.4"></script>



  
  

  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.4"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.4"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.4"></script>



  


  




	





  





  












  





  

  
  <script src="https://cdn1.lncld.net/static/js/av-core-mini-0.6.4.js"></script>
  <script>AV.initialize("NNEhOL0iOcflg8f1U3HUqiCq-gzGzoHsz", "7kSmkbbb3DktmHALlShDsBUF");</script>
  <script>
    function showTime(Counter) {
      var query = new AV.Query(Counter);
      var entries = [];
      var $visitors = $(".leancloud_visitors");

      $visitors.each(function () {
        entries.push( $(this).attr("id").trim() );
      });

      query.containedIn('url', entries);
      query.find()
        .done(function (results) {
          var COUNT_CONTAINER_REF = '.leancloud-visitors-count';

          if (results.length === 0) {
            $visitors.find(COUNT_CONTAINER_REF).text(0);
            return;
          }

          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.get('url');
            var time = item.get('time');
            var element = document.getElementById(url);

            $(element).find(COUNT_CONTAINER_REF).text(time);
          }
          for(var i = 0; i < entries.length; i++) {
            var url = entries[i];
            var element = document.getElementById(url);
            var countSpan = $(element).find(COUNT_CONTAINER_REF);
            if( countSpan.text() == '') {
              countSpan.text(0);
            }
          }
        })
        .fail(function (object, error) {
          console.log("Error: " + error.code + " " + error.message);
        });
    }

    function addCount(Counter) {
      var $visitors = $(".leancloud_visitors");
      var url = $visitors.attr('id').trim();
      var title = $visitors.attr('data-flag-title').trim();
      var query = new AV.Query(Counter);

      query.equalTo("url", url);
      query.find({
        success: function(results) {
          if (results.length > 0) {
            var counter = results[0];
            counter.fetchWhenSave(true);
            counter.increment("time");
            counter.save(null, {
              success: function(counter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(counter.get('time'));
              },
              error: function(counter, error) {
                console.log('Failed to save Visitor num, with error message: ' + error.message);
              }
            });
          } else {
            var newcounter = new Counter();
            /* Set ACL */
            var acl = new AV.ACL();
            acl.setPublicReadAccess(true);
            acl.setPublicWriteAccess(true);
            newcounter.setACL(acl);
            /* End Set ACL */
            newcounter.set("title", title);
            newcounter.set("url", url);
            newcounter.set("time", 1);
            newcounter.save(null, {
              success: function(newcounter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(newcounter.get('time'));
              },
              error: function(newcounter, error) {
                console.log('Failed to create');
              }
            });
          }
        },
        error: function(error) {
          console.log('Error:' + error.code + " " + error.message);
        }
      });
    }

    $(function() {
      var Counter = AV.Object.extend("Counter");
      if ($('.leancloud_visitors').length == 1) {
        addCount(Counter);
      } else if ($('.post-title-link').length > 1) {
        showTime(Counter);
      }
    });
  </script>



  

  

  
  

  

  

  

</body>
</html>
